import datetime
import json
import sys
import traceback
from operator import itemgetter

from fastapi import APIRouter
from pymongo.errors import DocumentTooLarge
from re_common.baselibrary.utils.basefile import BaseFile
from re_common.baselibrary.utils.core.mlamada import bools_string
from re_common.baselibrary.utils.core.requests_core import MsgCode

from apps.core.global_model import InputPlatformModel, IdModel
from apps.core.m_route import ContextIncludedRoute
from apps.core.return_info import ReturnInfo, SUCCESS, InputInfoModel
from apps.crawler_platform.core_api.models import SaveMongoModel, SaveModel, SelectMongoModel, ResetMongoModel, \
    SavePathModel
from apps.crawler_platform.util.nosqlhelper import NOSQLHelper
from apps.sql_app.mmongodb import Coll

router = APIRouter(route_class=ContextIncludedRoute)


@router.post("/mongo_api/save_mongo")
async def save_mongo_try(input: InputPlatformModel[SaveMongoModel]):
    from settings import get_settings
    from app_main import app
    sets = get_settings()
    """
    此方法为保存下载数据至mongo,逻辑大致如下:
    1.根据id生成规则组装id,查询mongo,若不存在,则保存为新纪录,若存在,进行更新操作
    2.mongodb里数据结构主要字段为
        new_data{}:新数据缓存字段,根据传入的data_op来判定是否将传入的data字段更新还是覆盖,data_op默认为0表更新
        latest_data{}:历史最后更新数据缓存字段,它等于history列表里日期最近的一条
        history[{}...]:历史数据保存字段,其大小由his_cnt字段决定
        当传入is_his='True'时,表明需要将new_data更新至history列表并且更新latest_data字段
    :param input:para{"id_list":["hh","jj"],"hh":"test","jj":"tag"},id_list为id组合顺序
    :param data:data{"down_data":{...},"sql_data":{...}} down_data：网页下载数据，sql_data：mysql 任务查询结果
    :return:
    """
    return_info = ReturnInfo()
    ids = input.data.id
    table = input.data.table
    para = input.data.para
    id_list = para["id_list"]
    is_his = input.data.is_his
    his_cnt = input.data.his_cnt
    data = input.data.data
    data_op = input.data.data_op
    id_ = "_".join([str(para[tmp]) for tmp in id_list])
    mongo_conn = Coll.get_table_conn(table)  # 根据table名取得mongo链接
    cjip = await NOSQLHelper.find_one(mongo_conn, {"_id": id_})
    down_time = str(datetime.datetime.now())
    down_date = datetime.datetime.now().strftime("%Y%m%d")
    is_path_error = False
    if cjip:
        if "path" in cjip.keys():
            try:
                async with app.state.aioboto3.client('s3', endpoint_url=sets.AWS_LOCAL_ENDPOINT_URL) as s3:
                    s3_ob = await s3.get_object(Bucket=sets.AWS_LOCAL_BUCKET, Key=cjip["path"])
                    cjip = json.loads((await s3_ob["Body"].read()).decode("utf-8"))
            except:
                rs_del = await NOSQLHelper.delete_many(mongo_conn, {"_id": id_})
                is_path_error = True
        if not is_path_error:
            new_data = cjip.get("new_data",{})
            history = cjip.get("history",[])
            # 下面为用传入的data根据data_op决定是更新还是覆盖new_data
            if len(new_data) != 0:
                if data_op == 0:
                    new_data.update(data)
                else:
                    new_data = data
            else:
                new_data = data
            new_data["down_date"] = down_date
            new_data["down_time"] = down_time
            if is_his == 'True':
                his_item = next((item for item in history if item.get("down_date") == down_date),
                                "")  # 得到history列表里当日的item
                if his_item != "":  # 当日已经保存过
                    if data_op == 0:  # 用new_data更新history里当日的item
                        his_item.update(new_data)
                        last_data = his_item
                    else:  # 用new_data覆盖history里当日的item
                        history.remove(his_item)
                        history.append(new_data)
                        last_data = new_data
                else:  # 当日第一次保存
                    if len(history) >= his_cnt:  # 若history里历史记录数量超过his_cnt,则删除日期最远的一条
                        history.sort(key=itemgetter("down_date"), reverse=False)
                        history.pop(0)
                    last_data = new_data
                    history.append(last_data)
                    history.sort(key=itemgetter("down_date"), reverse=True)
                cjip["last_data"] = last_data
                new_data = {}
            dicts = SaveModel.parse_obj(cjip).dict()
            dicts["new_data"] = new_data
            dicts["history"] = history
            dicts["update_time"] = down_time
            dicts["down_date"] = down_time
            dicts["_id"] = id_
            save = SaveModel.parse_obj(dicts)
            rinfo, result = await saveToLocal(1, mongo_conn, save)
            if isinstance(rinfo, ReturnInfo):  # 说明文件超出大小,要保存本地,且保存本地的时候出现问题了
                return rinfo.todict()
            if "path" in cjip.keys() and rinfo == False:  # 原来存本地文件,现在存mongo,则把本地文件删除
                BaseFile.remove_file(cjip["path"])
    if not cjip or is_path_error:
        last_data = {}
        history = []
        data["down_date"] = down_date
        data["down_time"] = down_time
        if is_his == 'False':
            new_data = data
        else:
            last_data = data
            history.append(last_data)
            new_data = {}
        save = SaveModel(
            _id=id_,
            para=para,
            last_data=last_data,
            new_data=new_data,
            history=history,
            update_time=str(datetime.datetime.now()),
            create_time=str(datetime.datetime.now())
        )
        if not cjip:
            rinfo, result = await saveToLocal(0, mongo_conn, save)
        else:
            rinfo, result = await saveToLocal(1, mongo_conn, save)
        if isinstance(rinfo, ReturnInfo):
            return rinfo.todict()
        if result.inserted_id != id_:
            return_info.status = bools_string(False)
            return_info.msg_code = MsgCode.MONGO_ERROR
            return_info.msg = "save_mongo接口插入mongo失败"
            return_info.data = {}
            return return_info.todict()
    return_info.status = SUCCESS
    return_info.msg_code = 200
    return_info.msg = "save_mongo接口保存到mongodb成功"
    return_info.data = ""
    return return_info.todict()


async def saveToLocal(op, mongo_conn, save):
    id_ = save.id
    is_local = False
    try:
        if op == 0:
            save_dict = save.dict(by_alias=True, exclude_none=True)
            # result = await mongo_conn.insert_one(save_dict)
            result = await NOSQLHelper.insert_one(mongo_conn, save_dict, list_exception_name=["DocumentTooLarge"])
        else:
            save_dict = save.dict(by_alias=True, exclude={"id"})
            # result = await mongo_conn.replace_one({'_id': id_}, save_dict)
            result = await NOSQLHelper.replace_one_substitute(mongo_conn, {'_id': id_}, save_dict, upsert=True,
                                                              list_exception_name=["DocumentTooLarge"])
    except DocumentTooLarge:
        is_local = True
    if not is_local:  # 如果没有爆出大小超出异常,则是保存到mongo里的,直接返回
        return is_local, result
    # sets = get_settings()
    # save_cnt = 0
    # while save_cnt < sets.MONGO_SAVE_TRY_CNT:
    #     try:
    #         save_path = BaseFile.get_new_path(sets.MONGO_BASE_DIR, sets.MONGO_SAVE_PATH)
    #         BaseDir.create_dir(save_path)
    #         save_path = BaseFile.get_new_path(save_path, "{}.big_json".format(id_))
    #         BaseFile.single_write_file(save_path, json.dumps(save_dict, ensure_ascii=False))
    #         if BaseFile.is_file_exists(save_path):
    #             break
    #         save_cnt += 1
    #     except:
    #         save_cnt += 1

    bools, save_path = await NOSQLHelper.save_gt_16M_to_minio(id_, save_dict)
    if bools:
        return_info = None
    else:
        return_info = ReturnInfo()
        return_info.status = bools_string(False)
        return_info.msg_code = MsgCode.SERVER_ERROR
        from app_main import sets
        return_info.msg = "保存数据至本地磁盘失败,重复次数:{}".format(sets.MONGO_SAVE_TRY_CNT)
        return_info.data = {}
    save = SavePathModel(
        _id=id_,
        path=save_path,
        update_time=str(datetime.datetime.now()),
        create_time=str(datetime.datetime.now())
    )
    if op == 0:
        save_dict = save.dict(by_alias=True, exclude_none=True)
        # result = await mongo_conn.insert_one(save_dict)
        result = await NOSQLHelper.insert_one(mongo_conn, save_dict)
    else:
        save_dict = save.dict(by_alias=True, exclude={"id"})
        # result = await mongo_conn.replace_one({'_id': id_}, save_dict)
        result = await NOSQLHelper.replace_one_substitute(mongo_conn, {'_id': id_}, save_dict, upsert=True)
    return return_info, result


def get_size(obj, seen=None):
    size = sys.getsizeof(obj)
    if seen is None:
        seen = set()
    obj_id = id(obj)
    if obj_id in seen:
        return 0
    seen.add(obj_id)
    if isinstance(obj, dict):
        size += sum([get_size(v, seen) for v in obj.values()])
        size += sum([get_size(k, seen) for k in obj.keys()])
    elif hasattr(obj, '__dict__'):
        size += get_size(obj.__dict__, seen)
    elif hasattr(obj, '__iter__') and not isinstance(obj, (str, bytes, bytearray)):
        size += sum([get_size(i, seen) for i in obj])
    return size


@router.post("/mongo_api/save_mongo_etl")
async def save_mongo_etl(input: InputPlatformModel[SaveMongoModel]):
    return_info = ReturnInfo()
    table = input.data.table
    data = input.data.data
    _id = data.get("_id")
    if not _id:
        _id = data["lngid"]
        data["_id"] = _id
    mongo_conn = Coll.get_table_conn(table)  # 根据table名取得mongo链接
    cjip = await NOSQLHelper.find_one(mongo_conn, {"_id": _id})
    if cjip:
        # 20251105 防止cnki重刷数据url改变
        if data['sub_db_id'] == '00002':
            if 'abstract?v=' in cjip.get('provider_url', '') and data['title'] == cjip['title']:
                data['provider_url'] = cjip['provider_url']
        # result = await mongo_conn.replace_one({'_id': lngid}, data)
        result = await NOSQLHelper.replace_one_substitute(mongo_conn, {'_id': _id}, data, upsert=True)
    else:
        # result = await mongo_conn.insert_one(data)
        result = await NOSQLHelper.insert_one(mongo_conn, data)
        if result.inserted_id != _id:
            return_info.status = bools_string(False)
            return_info.msg_code = MsgCode.MONGO_ERROR
            return_info.msg = "插入mongo失败"
            return_info.data = {}
            return return_info.todict()
    return_info.status = SUCCESS
    return_info.msg_code = 200
    return_info.msg = "保存到mongodb成功"
    return_info.data = ""
    return return_info.todict()


@router.post("/mongo_api/local_etl_insert")
async def local_etl_insert(input: InputPlatformModel[SaveMongoModel]):
    return_info = ReturnInfo()
    table = input.data.table
    data = input.data.data
    _id = data.get("_id")
    if not _id:
        _id = data["lngid"]
        data["_id"] = _id
    mongo_conn = Coll.get_table_conn(table)  # 根据table名取得mongo链接
    cjip = await NOSQLHelper.find_one(mongo_conn, {"_id": _id})
    status = SUCCESS
    msg_code = 200
    msg = "保存到mongodb成功"
    redict = {}
    if not cjip:
        result = await NOSQLHelper.insert_one(mongo_conn, data)
        if result.inserted_id != _id:
            status = bools_string(False)
            msg_code = MsgCode.MONGO_ERROR
            msg = "插入mongo失败"
            redict = return_info.todict()
    else:
        msg = "mongo已存在,跳过"
    return_info.status = status
    return_info.msg_code = msg_code
    return_info.msg = msg
    return_info.data = redict
    return return_info.todict()


@router.post("/mongo_api/local_etl_delete")
async def local_etl_delete(input: InputInfoModel[IdModel]):
    return_info = ReturnInfo()
    table = input.data.table
    data = input.data.data
    _id = data.get("_id")
    mongo_conn = Coll.get_table_conn(table)  # 根据table名取得mongo链接
    # cjip = await NOSQLHelper.find_one(mongo_conn, {"_id": _id})
    status = SUCCESS
    msg_code = 200
    msg = "删除mongodb成功"
    redict = {}
    result = await NOSQLHelper.delete_many(mongo_conn, {"_id": _id})
    if result <= 0:
        status = bools_string(False)
        msg_code = MsgCode.MONGO_ERROR
        msg = "删除mongo失败"
    return_info.status = status
    return_info.msg_code = msg_code
    return_info.msg = msg
    return_info.data = redict
    return return_info.todict()


@router.post("/mongo_api/reset_new_data")
async def reset_new_data_try(input: InputPlatformModel[ResetMongoModel]):
    return_info = ReturnInfo()
    from settings import get_settings
    from app_main import app
    sets = get_settings()
    id_ = input.data.id
    table = input.data.table
    mongo_conn = Coll.get_table_conn(table)  # 根据table名取得mongo链接
    # cjip = await mongo_conn.find_one({"_id": id_})
    cjip = await NOSQLHelper.find_one(mongo_conn, {"_id": id_})
    is_path_error = False
    if cjip:
        if "path" in cjip.keys():
            try:
                async with app.state.aioboto3.client('s3', endpoint_url=sets.AWS_LOCAL_ENDPOINT_URL) as s3:
                    s3_ob = await s3.get_object(Bucket=sets.AWS_LOCAL_BUCKET, Key=cjip["path"])
                    cjip = json.loads((await s3_ob["Body"].read()).decode("utf-8"))
            except:
                rs_del = await NOSQLHelper.delete_many(mongo_conn, {"_id": id_})
                is_path_error = True
        if not is_path_error:
            new_data = cjip["new_data"]
            last_data = cjip["last_data"]
            if len(new_data) == 0 and len(last_data) > 0:
                dicts = SaveModel.parse_obj(cjip).dict()
                dicts["new_data"] = last_data
                save = SaveModel.parse_obj(dicts)
                rinfo, result = await saveToLocal(1, mongo_conn, save)
                if isinstance(rinfo, ReturnInfo):  # 说明文件超出大小,要保存本地,且保存本地的时候出现问题了
                    return rinfo.todict()
                if "path" in cjip.keys() and rinfo == False:  # 原来存本地文件,现在存mongo,则把本地文件删除
                    BaseFile.remove_file(cjip["path"])
        else:
            return_info.status = bools_string(False)
            return_info.msg_code = MsgCode.MONGO_ERROR
            return_info.msg = "reset_new_data接口更新mongodb的new_data失败,new_data不为空或者latest_data为空"
            return_info.data = {}
            return return_info.todict()
    return_info.status = SUCCESS
    return_info.msg_code = 200
    return_info.msg = "reset_new_data接口更新mongodb的new_data成功"
    return_info.data = ""
    return return_info.todict()


@router.post("/mongo_api/select_mongo")
async def select_mongo(input: InputPlatformModel[SelectMongoModel]):
    """
      通用查询
      :param field:{"new_data.html":1,"last_data":1} 需要返回的字段值
      :param table:查询的表名
      :param and_param:{"_id":"001","para.task_name":"test"} 用and拼接的查询条件
      :param or_param:[{"_id":"001"},{"_id":"002"}]用or拼接的查询条件
      :param order:[("create_time":1)]排序字段
      :param limit:int,限制返回数量
      :return:
    """
    from settings import get_settings
    from app_main import app
    sets = get_settings()
    return_info = ReturnInfo()
    field = input.data.field
    table = input.data.table
    and_param = input.data.and_param
    or_param = input.data.or_param
    order = input.data.order
    limit = input.data.limit
    mongo_conn = Coll.get_table_conn(table)
    query = {}
    query.update(and_param)
    if len(or_param) > 0:
        query.update({
            "$or": or_param
        })
    if len(field) == 0:
        field = None
    else:
        field.update({"path": 1})
    # if len(query.keys()) == 1:
    #     rows = None
    #     row = await NOSQLHelper.find_one(mongo_conn, query, field)
    #     if row:
    #         rows = [row]
    # else:
    rows = await NOSQLHelper.find(mongo_conn, query, projection=field, limit=limit, sort=order)
    new_rows = []
    if rows:
        for row in rows:
            if "path" in row.keys():
                try:
                    async with app.state.aioboto3.client('s3', endpoint_url=sets.AWS_LOCAL_ENDPOINT_URL) as s3:
                        s3_ob = await s3.get_object(Bucket=sets.AWS_LOCAL_BUCKET, Key=row["path"])
                        row = json.loads((await s3_ob["Body"].read()).decode("utf-8"))
                except:
                    return_info.status = bools_string(False)
                    return_info.msg_code = MsgCode.SERVER_ERROR
                    return_info.msg = "获取本地磁盘存放数据失败"
                    return_info.data = {}
                    return return_info.todict()
            new_rows.append(row)
    return_info.status = SUCCESS
    return_info.msg_code = 200
    return_info.msg = "查询mongodb成功"
    return_info.data = new_rows
    return return_info.todict()
